//package com.nlp.visualization.crawler.downloader;
//
//import java.io.IOException;
//import java.nio.charset.Charset;
//import java.util.HashMap;
//import java.util.Map;
//import java.util.Set;
//
//import org.apache.commons.io.IOUtils;
//import org.apache.http.HttpResponse;
//import org.apache.http.client.methods.CloseableHttpResponse;
//import org.apache.http.client.methods.HttpUriRequest;
//import org.apache.http.impl.client.CloseableHttpClient;
//import org.apache.http.util.EntityUtils;
//import org.slf4j.Logger;
//import org.slf4j.LoggerFactory;
//
//import com.google.common.collect.Sets;
//
//import us.codecraft.webmagic.Page;
//import us.codecraft.webmagic.Request;
//import us.codecraft.webmagic.Site;
//import us.codecraft.webmagic.Task;
//import us.codecraft.webmagic.downloader.HttpClientDownloader;
//import us.codecraft.webmagic.downloader.HttpClientGenerator;
//import us.codecraft.webmagic.selector.PlainText;
//
///**
// * @author LXM
// * @Title: CharsetConfigDownloader
// * @Description: 用不同的编码集来爬取网页，避免乱码
// * @date 2018/3/4下午1:04
// */
//public class CharsetConfigDownloader extends HttpClientDownloader {
//    private Logger logger= LoggerFactory.getLogger(CharsetConfigDownloader.class);
//
//    private final Map<String, CloseableHttpClient> httpClients = new HashMap<String, CloseableHttpClient>();
//
//    private HttpClientGenerator httpClientGenerator = new HttpClientGenerator();
//
//    private CloseableHttpClient getHttpClient(Site site) {
//        if (site == null) {
//            return httpClientGenerator.getClient(null);
//        }
//        String domain = site.getDomain();
//        CloseableHttpClient httpClient = httpClients.get(domain);
//        if (httpClient == null) {
//            synchronized (this) {
//                httpClient = httpClients.get(domain);
//                if (httpClient == null) {
//                    httpClient = httpClientGenerator.getClient(site);
//                    httpClients.put(domain, httpClient);
//                }
//            }
//        }
//        return httpClient;
//    }
//
//    @Override
//    public Page download(Request request, Task task) {
//        Site site = null;
//        if (task != null) {
//            site = task.getSite();
//        }
//
//        Set<Integer> acceptStatCode;
//        String charset = null;
//        Map<String, String> headers = null;
//        if (site != null) {
//            acceptStatCode = site.getAcceptStatCode();
//            charset = site.getCharset();
//            headers = site.getHeaders();
//        } else {
//            acceptStatCode = Sets.newHashSet(200);
//        }
//        logger.info("downloading page {}", request.getUrl());
//        CloseableHttpResponse httpResponse = null;
//        int statusCode = 0;
//        try {
//            if (request.getExtra("_charset") != null) {
//                charset = request.getExtra("_charset").toString();
//            }
//            if (request.getExtra("_referer") != null) {
//                headers.put("Referer", request.getExtra("_referer").toString());
//            }
//            HttpUriRequest httpUriRequest = getHttpUriRequest(request, site, headers);
//            httpResponse = getHttpClient(site).execute(httpUriRequest);
//            statusCode = httpResponse.getStatusLine().getStatusCode();
//            request.putExtra(Request.STATUS_CODE, statusCode);
//            if (statusAccept(acceptStatCode, statusCode)) {
//                Page page = handleResponse(request, charset, httpResponse, task);
//                onSuccess(request);
//                return page;
//            } else {
//                logger.warn("code error " + statusCode + "\t" + request.getUrl());
//                return null;
//            }
//        } catch (IOException e) {
//            logger.warn("download page " + request.getUrl() + " error", e);
//            if (site.getCycleRetryTimes() > 0) {
//                return addToCycleRetry(request, site);
//            }
//            onError(request);
//            return null;
//        } finally {
//            request.putExtra(Request.STATUS_CODE, statusCode);
//            try {
//                if (httpResponse != null) {
//                    //ensure the connection is released back to pool
//                    EntityUtils.consume(httpResponse.getEntity());
//                }
//            } catch (IOException e) {
//                logger.warn("close response fail", e);
//            }
//        }
//    }
//
//
//    @Override
//    protected String getContent(String charset, HttpResponse httpResponse) throws IOException {
//        if (charset == null) {
//            byte[] contentBytes = IOUtils.toByteArray(httpResponse.getEntity().getContent());
//            String htmlCharset = getHtmlCharset(httpResponse, contentBytes);
//            if (htmlCharset != null) {
//                return new String(contentBytes, htmlCharset);
//            } else {
//                logger.warn("Charset autodetect failed, use {} as charset. Please specify charset in Site.setCharset()", Charset.defaultCharset());
//                return new String(contentBytes);
//            }
//        } else {
//            String contentStr = IOUtils.toString(httpResponse.getEntity().getContent(), charset);
//            return contentStr;    //new String(contentStr.getBytes(charset),"utf-8");
//        }
//    }
//}
